{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /home/jhon/data/train-images-idx3-ubyte.gz\n",
      "Extracting /home/jhon/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /home/jhon/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /home/jhon/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = '/home/jhon/data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义卷积参数\n",
    "INPUT_NODE = 784\n",
    "OUTPUT_NODE = 10\n",
    "\n",
    "IMAGE_SIZE = 28\n",
    "NUM_CHANNELS = 1\n",
    "NUM_LABELS = 10\n",
    "\n",
    "CONV1_DEEP = 32\n",
    "CONV1_SIZE = 5\n",
    "\n",
    "CONV2_DEEP = 64\n",
    "CONV2_SIZE = 5\n",
    "\n",
    "FC_SIZE = 512\n",
    "\n",
    "BATCH_SIZE = 100\n",
    "LEARNING_RATE_BASE = 0.01\n",
    "LEARNING_RATE_DECAY = 0.95\n",
    "REGULARIZATION_RATE = 0.0001\n",
    "TRAINING_STEPS = 3000\n",
    "MOVING_AVERAGE_DECAY = 0.99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inference(input_tensor, train, regularizer):\n",
    "    with tf.variable_scope('layer1-conv1', reuse=tf.AUTO_REUSE):\n",
    "        conv1_weights = tf.get_variable(\n",
    "            \"weight\", [CONV1_SIZE, CONV1_SIZE, NUM_CHANNELS, CONV1_DEEP],\n",
    "        initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(conv1_weights))\n",
    "        conv1_biases = tf.get_variable(\"bias\", [CONV1_DEEP], initializer=tf.constant_initializer(0.0))\n",
    "        conv1 = tf.nn.conv2d(input_tensor, conv1_weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "        relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))\n",
    "\n",
    "    with tf.name_scope(\"layer2-pool1\"):\n",
    "        pool1 = tf.nn.max_pool(relu1, ksize = [1,2,2,1],strides=[1,2,2,1],padding=\"SAME\")\n",
    "\n",
    "    with tf.variable_scope(\"layer3-conv2\", reuse=tf.AUTO_REUSE):\n",
    "        conv2_weights = tf.get_variable(\n",
    "            \"weight\", [CONV2_SIZE, CONV2_SIZE, CONV1_DEEP, CONV2_DEEP],\n",
    "        initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(conv2_weights))\n",
    "        conv2_biases = tf.get_variable(\"bias\", [CONV2_DEEP], initializer=tf.constant_initializer(0.0))\n",
    "        conv2 = tf.nn.conv2d(pool1, conv2_weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "        relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))\n",
    "\n",
    "    with tf.name_scope(\"layer4-pool2\",):\n",
    "        pool2 = tf.nn.max_pool(relu2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        pool_shape = pool2.get_shape().as_list()\n",
    "        nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]\n",
    "        reshaped = tf.reshape(pool2, [pool_shape[0], nodes])\n",
    "\n",
    "    with tf.variable_scope('layer5-fc1', reuse=tf.AUTO_REUSE):\n",
    "        fc1_weights = tf.get_variable(\"weight\", [nodes, FC_SIZE],\n",
    "        initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(fc1_weights))\n",
    "        fc1_biases = tf.get_variable(\"bias\", [FC_SIZE], initializer=tf.constant_initializer(0.1))\n",
    "\n",
    "        fc1 = tf.nn.relu(tf.matmul(reshaped, fc1_weights) + fc1_biases)\n",
    "        if train: fc1 = tf.nn.dropout(fc1, 0.5)\n",
    "\n",
    "    with tf.variable_scope('layer6-fc2', reuse=tf.AUTO_REUSE):\n",
    "        fc2_weights = tf.get_variable(\"weight\", [FC_SIZE, NUM_LABELS],\n",
    "        initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(fc2_weights))\n",
    "        fc2_biases = tf.get_variable(\"bias\", [NUM_LABELS], initializer=tf.constant_initializer(0.1))\n",
    "        logit = tf.matmul(fc1, fc2_weights) + fc2_biases\n",
    "\n",
    "    return logit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义输出为4维矩阵的placeholder\n",
    "x = tf.placeholder(tf.float32, [\n",
    "            BATCH_SIZE,\n",
    "            IMAGE_SIZE,\n",
    "            IMAGE_SIZE,\n",
    "            NUM_CHANNELS],\n",
    "        name='x-input')\n",
    "y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')\n",
    "    \n",
    "regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)\n",
    "y = inference(x,False,regularizer)\n",
    "global_step = tf.Variable(0, trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, total loss: 1.013485\n",
      "0.91\n",
      "step 200, total loss: 0.921382\n",
      "0.96\n",
      "step 300, total loss: 0.895795\n",
      "0.95\n",
      "step 400, total loss: 0.842296\n",
      "0.95\n",
      "step 500, total loss: 0.841442\n",
      "0.95\n",
      "step 600, total loss: 0.791217\n",
      "0.96\n",
      "step 700, total loss: 0.734451\n",
      "0.99\n",
      "step 800, total loss: 0.841583\n",
      "0.97\n",
      "step 900, total loss: 0.801863\n",
      "0.99\n",
      "step 1000, total loss: 0.880363\n",
      "0.94\n",
      "After 1000 training steps, validation accuracy using average model is 0.94 \n",
      "step 1100, total loss: 0.758991\n",
      "0.98\n",
      "step 1200, total loss: 0.715594\n",
      "1.0\n",
      "step 1300, total loss: 0.721087\n",
      "0.99\n",
      "step 1400, total loss: 0.692672\n",
      "1.0\n",
      "step 1500, total loss: 0.765560\n",
      "0.98\n",
      "step 1600, total loss: 0.728207\n",
      "1.0\n",
      "step 1700, total loss: 0.766717\n",
      "0.97\n",
      "step 1800, total loss: 0.726179\n",
      "0.98\n",
      "step 1900, total loss: 0.730922\n",
      "0.98\n",
      "step 2000, total loss: 0.738835\n",
      "0.98\n",
      "After 2000 training steps, validation accuracy using average model is 0.98 \n",
      "step 2100, total loss: 0.729849\n",
      "0.99\n",
      "step 2200, total loss: 0.705197\n",
      "0.98\n",
      "step 2300, total loss: 0.718586\n",
      "0.99\n",
      "step 2400, total loss: 0.688875\n",
      "0.99\n",
      "step 2500, total loss: 0.699700\n",
      "1.0\n",
      "step 2600, total loss: 0.767727\n",
      "0.97\n",
      "step 2700, total loss: 0.733851\n",
      "0.99\n",
      "step 2800, total loss: 0.695876\n",
      "0.99\n",
      "step 2900, total loss: 0.683317\n",
      "1.0\n",
      "step 3000, total loss: 0.742941\n",
      "0.98\n",
      "After 3000 training steps, validation accuracy using average model is 0.98 \n"
     ]
    }
   ],
   "source": [
    "# 定义损失函数、学习率、滑动平均操作以及训练过程。\n",
    "variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)\n",
    "variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))\n",
    "cross_entropy_mean = tf.reduce_mean(cross_entropy)\n",
    "loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))\n",
    "#学习率采用指数衰减的方式进行调整\n",
    "learning_rate = tf.train.exponential_decay(\n",
    "    LEARNING_RATE_BASE,\n",
    "    global_step,\n",
    "    mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY,\n",
    "    staircase=True)\n",
    "\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "with tf.control_dependencies([train_step, variables_averages_op]):\n",
    "    train_op = tf.no_op(name='train')\n",
    "    \n",
    "#正确率\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "        \n",
    "# 初始化TensorFlow持久化类。\n",
    "saver = tf.train.Saver()\n",
    "with tf.Session() as sess:\n",
    "    tf.global_variables_initializer().run()\n",
    "    for i in range(TRAINING_STEPS):\n",
    "        xs, ys = mnist.train.next_batch(BATCH_SIZE)\n",
    "\n",
    "        reshaped_xs = np.reshape(xs, (\n",
    "                BATCH_SIZE,\n",
    "                IMAGE_SIZE,\n",
    "                IMAGE_SIZE,\n",
    "                NUM_CHANNELS))\n",
    "        _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: reshaped_xs, y_: ys})\n",
    "\n",
    "        if (i+1) % 100 == 0:\n",
    "            print('step %d, total loss: %f' %(i+1,loss_value))\n",
    "    # Test trained model\n",
    "            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            print(sess.run(accuracy, feed_dict={x: reshaped_xs, y_: ys}))\n",
    "       \n",
    "        if (i+1) % 1000 == 0:\n",
    "            validate_acc = sess.run(accuracy, feed_dict = {x: reshaped_xs, y_: ys})\n",
    "            print(\"After %d training steps, validation accuracy using average model is %g \" % ((i+1), validate_acc))\n",
    "            "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
